package com.chance.cc.crawler.development.scripts.gmw;

import com.alibaba.fastjson.parser.deserializer.AbstractDateDeserializer;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.crawler.CrawlerResultTags;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.ParseException;
import java.util.*;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.interaction;

public class GmwCrawlerScript extends CrawlerCommonScript {
    private Logger log= LoggerFactory.getLogger(GmwCrawlerScript.class);

    @Override
    public String domain() {
        return "gmw";
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        return true;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular("https://www.gmw.cn/");
        addUrlRegular("https://\\S*.gmw.cn/");
        addUrlRegular("https://\\S*.gmw.cn/node_\\S*.htm");
        addUrlRegular("https://\\S*.gmw.cn/\\d*-\\d*/\\d*/content_\\d*.htm");
        addUrlRegular("https://\\S*.gmw.cn/\\S*/\\d*-\\d*/\\d*/content_\\d*.htm");
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks=new ArrayList<>();
        String requestUrl=httpPage.getRequest().getUrl();
        if(requestUrl.matches("https://www.gmw.cn/")){
            List<String> listUrl=httpPage.getHtml().xpath("//div[@class=\"m_navs page_width\"]/ul/li/a/@href").all();
            for(String itemUrl:listUrl){
                itemUrl="https:"+itemUrl;
                if(itemUrl.matches("https://(politics|tech|edu|economy).gmw.cn/")){
                    CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(itemUrl)
                            .httpUrl(itemUrl)
                            .releaseTime(System.currentTimeMillis())
                            .build();

                    parsedLinks.add(turnPageRequest);
                }
            }
        }else if(requestUrl.matches("https://\\S*.gmw.cn/")){

                    if(requestUrl.matches("https://politics.gmw.cn/")){
                        List<String> listUrl=httpPage.getHtml().xpath("//div[@class=\"w1000\"]/a/@href").all();
                        for(String itemUrl:listUrl){
                            itemUrl="https:"+itemUrl;
                            if(itemUrl.matches("https://politics.gmw.cn/node_(9844|9840|9831).htm")){
                                CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                                        .itemPageRequest(crawlerRequestRecord)
                                        .recordKey(itemUrl)
                                        .httpUrl(itemUrl)
                                        .releaseTime(System.currentTimeMillis())
                                        .build();

                                parsedLinks.add(turnPageRequest);
                            }
                        }
                    }else if(requestUrl.matches("https://tech.gmw.cn/")){
                        List<String> listUrl=httpPage.getHtml().xpath("//a[@class=\"more_load\"]/@href").all();
                        for(String itemUrl:listUrl){
                            itemUrl="https:"+itemUrl;
                            if(itemUrl.matches("https://tech.gmw.cn/node_10249.htm")){
                                CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                                        .itemPageRequest(crawlerRequestRecord)
                                        .recordKey(itemUrl)
                                        .httpUrl(itemUrl)
                                        .releaseTime(System.currentTimeMillis())
                                        .build();

                                parsedLinks.add(turnPageRequest);
                            }
                        }
                    }else if(requestUrl.matches("https://edu.gmw.cn/")){
                        List<String> listUrl=httpPage.getHtml().xpath("//div[@class=\"nav\"]/div/ul/li/a/@href").all();
                        for(String itemUrl:listUrl){
                            itemUrl="https:"+itemUrl;
                            if(itemUrl.matches("https://edu.gmw.cn/node_(9757|10602).htm")){
                                CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                                        .itemPageRequest(crawlerRequestRecord)
                                        .recordKey(itemUrl)
                                        .httpUrl(itemUrl)
                                        .releaseTime(System.currentTimeMillis())
                                        .build();

                                parsedLinks.add(turnPageRequest);
                            }
                        }
                    }else if(requestUrl.matches("https://economy.gmw.cn/")){
                        List<String> listUrl=httpPage.getHtml().xpath("//div[@id=\"nav_nengyuan\"]/ul/li/a/@href").all();
                        for(String itemUrl:listUrl){
                            itemUrl="https://economy.gmw.cn/"+itemUrl;
                            if(itemUrl.matches("https://economy.gmw.cn/node_59269.htm")){
                                itemUrl=itemUrl.substring(0,itemUrl.lastIndexOf("_")+1)+"12470.htm";
                            }
                            if(itemUrl.matches("https://economy.gmw.cn/node_(9141|12470|21787|8971).htm")){
                                CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                                        .itemPageRequest(crawlerRequestRecord)
                                        .recordKey(itemUrl)
                                        .httpUrl(itemUrl)
                                        .releaseTime(System.currentTimeMillis())
                                        .build();

                                parsedLinks.add(turnPageRequest);
                            }
                        }
                    }

        }else if(requestUrl.matches("https://edu.gmw.cn/node_\\S*.htm")){
            String nextPageUrl = null;
            //判断是否为第一页
            if (requestUrl.matches("https://edu.gmw\\.cn/node_\\d*\\.htm")){
                nextPageUrl = requestUrl.replace(".htm","_2.htm");
            }else {
                int currentPage = Integer.parseInt(requestUrl.substring(requestUrl.lastIndexOf("_") + 1).split("\\.")[0]);
                nextPageUrl = requestUrl.substring(0 , requestUrl.lastIndexOf("_") + 1) + (currentPage + 1) + ".htm";
            }

            CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .build();

            parsedLinks.add(turnPageRequest);



            List<Selectable> nodes=httpPage.getHtml().xpath("//ul[@class=\"channel-newsGroup\"]/li").nodes();
            for(Selectable node:nodes){
                String itemUrl=node.xpath("./span/a/@href").get();
                if(StringUtils.isBlank(itemUrl)){
                    continue;
                }
                if(!itemUrl.contains("https")){
                    itemUrl="https://edu.gmw.cn/"+itemUrl;
                }

                String pubTime = node.xpath(".//span[@class=\"channel-newsTime\"]").get();
                try {
                    long releaseTime = DateUtils.parseDate(pubTime, "yyyy-MM-dd").getTime();

                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(itemUrl)
                            .httpUrl(itemUrl)
                            .needParsed(false)
                            .releaseTime(releaseTime)
                            .resultLabelTag(article)
                            .build();
                    parsedLinks.add(itemRecord);
                } catch (ParseException e) {
                    e.printStackTrace();
                }

            }
        }else if(requestUrl.matches("https://politics.gmw.cn/node_\\S*.htm")){
            String nextPageUrl = null;
            //判断是否为第一页
            if (requestUrl.matches("https://politics.gmw\\.cn/node_\\d*\\.htm")){
                nextPageUrl = requestUrl.replace(".htm","_2.htm");
            }else {
                int currentPage = Integer.parseInt(requestUrl.substring(requestUrl.lastIndexOf("_") + 1).split("\\.")[0]);
                nextPageUrl = requestUrl.substring(0 , requestUrl.lastIndexOf("_") + 1) + (currentPage + 1) + ".htm";
            }

            CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .build();

            parsedLinks.add(turnPageRequest);

            List<Selectable> nodes=httpPage.getHtml().xpath("//ul[@class=\"channel-newsGroup\"]/li").nodes();
            for(Selectable node:nodes){
                String itemUrl=node.xpath("./a/@href").get();
                if(StringUtils.isBlank(itemUrl)){
                    continue;
                }

                    itemUrl="https://politics.gmw.cn/"+itemUrl;

                String pubTime = node.xpath(".//span[@class=\"channel-newsTime\"]").get();
                try {
                    long releaseTime = DateUtils.parseDate(pubTime, "yyyy-MM-dd").getTime();

                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(itemUrl)
                            .httpUrl(itemUrl)
                            .needParsed(false)
                            .releaseTime(releaseTime)
                            .resultLabelTag(article)
                            .build();
                    parsedLinks.add(itemRecord);
                } catch (ParseException e) {
                    e.printStackTrace();
                }

            }
        }else if(requestUrl.matches("https://economy.gmw.cn/node_\\S*.htm")){
            String nextPageUrl = null;
            //判断是否为第一页
            if (requestUrl.matches("https://economy.gmw\\.cn/node_\\d*\\.htm")){
                nextPageUrl = requestUrl.replace(".htm","_2.htm");
            }else {
                int currentPage = Integer.parseInt(requestUrl.substring(requestUrl.lastIndexOf("_") + 1).split("\\.")[0]);
                nextPageUrl = requestUrl.substring(0 , requestUrl.lastIndexOf("_") + 1) + (currentPage + 1) + ".htm";
            }

            CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .build();

            parsedLinks.add(turnPageRequest);

            List<Selectable> nodes=httpPage.getHtml().xpath("//ul[@class=\"channel-newsGroup\"]/li").nodes();
            for(Selectable node:nodes){
                String itemUrl=node.xpath("./a/@href").get();
                if(StringUtils.isBlank(itemUrl)){
                    continue;
                }

                    itemUrl="https://economy.gmw.cn/"+itemUrl;

                String pubTime = node.xpath(".//span[@class=\"channel-newsTime\"]").get();
                try {
                    long releaseTime = DateUtils.parseDate(pubTime, "yyyy-MM-dd").getTime();

                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(itemUrl)
                            .httpUrl(itemUrl)
                            .needParsed(false)
                            .releaseTime(releaseTime)
                            .resultLabelTag(article)
                            .build();
                    parsedLinks.add(itemRecord);
                } catch (ParseException e) {
                    e.printStackTrace();
                }

            }

        }else if(requestUrl.matches("https://tech.gmw.cn/node_\\S*.htm")){
            String nextPageUrl = null;
            //判断是否为第一页
            if (requestUrl.matches("https://tech.gmw\\.cn/node_\\d*\\.htm")){
                nextPageUrl = requestUrl.replace(".htm","_2.htm");
            }else {
                int currentPage = Integer.parseInt(requestUrl.substring(requestUrl.lastIndexOf("_") + 1).split("\\.")[0]);
                nextPageUrl = requestUrl.substring(0 , requestUrl.lastIndexOf("_") + 1) + (currentPage + 1) + ".htm";
            }

            CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .build();

            parsedLinks.add(turnPageRequest);



            List<Selectable> nodes=httpPage.getHtml().xpath("//div[@class=\"main_list\"]/div").nodes();
            for(Selectable node:nodes){
                String itemUrl=node.xpath("./a/@href").get();

                if(StringUtils.isBlank(itemUrl)){
                    continue;
                }

                String pubTime = node.xpath(".//span[@class=\"time\"]").get();

                if (!pubTime.contains("年")) {
                    Calendar date = Calendar.getInstance();
                    String year = String.valueOf(date.get(Calendar.YEAR));
                    pubTime = year + "-" + pubTime;
                }

                try {
                    long releaseTime = DateUtils.parseDate(pubTime, "yyyy-MM-dd HH:mm").getTime();

                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(itemUrl)
                            .httpUrl(itemUrl)
                            .needParsed(false)
                            .releaseTime(releaseTime)
                            .resultLabelTag(article)
                            .build();
                    parsedLinks.add(itemRecord);
                } catch (ParseException e) {
                    e.printStackTrace();
                }

            }
        }



        return parsedLinks;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        CrawlerResultTags crawlerResultTags = crawlerRecord.tagsCreator().resultTags();
        if (crawlerResultTags.hasDataType(article)) {
            crawlerDataList.add(washArticle(crawlerRecord, page));

        }

        if (crawlerResultTags.hasDataType(interaction)) {
            crawlerDataList.add(washInteraction(crawlerRecord, page));
        }

        return crawlerDataList;
    }

    public CrawlerData washArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        Map<String,String> article = new HashMap<>();
        String requestUrl=httpPage.getRequest().getUrl();

        if(requestUrl.matches("https://politics.gmw.cn/\\d*-\\d*/\\d*/content_\\d*.htm")){
            try {

                List<String> texts = httpPage.getHtml().xpath("//div[@class=\"u-mainText\"]/p/text()|//div[@class=\"u-mainText\"]/p/span/text()").all();
                String releaseTimeStr = httpPage.getHtml().xpath("//span[@class=\"m-con-time\"]/text()").get();

                String author=httpPage.getHtml().xpath("//span[@class=\"m-con-source\"]/a/text()").get();
                if(author==null){
                    author=httpPage.getHtml().xpath("//span[@class=\"m-con-source\"]/text()").get();
                    author=author.substring(author.indexOf("：")+1);
                }

                StringBuffer conents = new StringBuffer();
                for (String text : texts) {
                    conents.append(text).append(" ");
                }

                String itemUrl = httpPage.getRequest().getUrl();
                String articleKey = itemUrl.substring(itemUrl.lastIndexOf("_") + 1).split("\\.")[0];

                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRequestRecord, httpPage)
                        .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                        .url(itemUrl)
                        .releaseTime(DateUtils.parseDate(releaseTimeStr,"yyyy-MM-dd HH:mm").getTime())
                        .addContentKV("content",conents.toString().trim())
                        .addContentKV("title",httpPage.getHtml().xpath("//h1[@class=\"u-title\"]/text()").get())
                        .addContentKV("author",author)
                        .build();
                return crawlerData;
            } catch (ParseException e) {
                log.error(e.getMessage(),e);
            }
        }else if(requestUrl.matches("https://economy.gmw.cn/\\d{4}-\\d*/\\d*/content_\\d*.htm")){
            try {

                List<String> texts = httpPage.getHtml().xpath("//div[@class=\"u-mainText\"]/p/text()").all();
                String releaseTimeStr = httpPage.getHtml().xpath("//span[@class=\"m-con-time\"]/text()").get();

                StringBuffer conents = new StringBuffer();
                for (String text : texts) {
                    conents.append(text).append(" ");
                }

                String itemUrl = httpPage.getRequest().getUrl();
                String articleKey = itemUrl.substring(itemUrl.lastIndexOf("_") + 1).split("\\.")[0];

                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRequestRecord, httpPage)
                        .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                        .url(itemUrl)
                        .releaseTime(DateUtils.parseDate(releaseTimeStr,"yyyy-MM-dd HH:mm").getTime())
                        .addContentKV("content",conents.toString().trim())
                        .addContentKV("title",httpPage.getHtml().xpath("//h1[@class=\"u-title\"]/text()").get())
                        .addContentKV("author",httpPage.getHtml().xpath("//span[@class=\"m-con-source\"]/a/text()").get())
                        .build();
                return crawlerData;
            } catch (ParseException e) {
                log.error(e.getMessage(),e);
            }
        }else if(requestUrl.matches("https://tech.gmw.cn/\\d*-\\d*/\\d*/content_\\d*.htm")){
            try {

                List<String> texts = httpPage.getHtml().xpath("//div[@class=\"u-mainText\"]/p/text()").all();
                String releaseTimeStr = httpPage.getHtml().xpath("//span[@class=\"m-con-time\"]/text()").get();

                StringBuffer conents = new StringBuffer();
                for (String text : texts) {
                    conents.append(text).append(" ");
                }

                String itemUrl = httpPage.getRequest().getUrl();
                String articleKey = itemUrl.substring(itemUrl.lastIndexOf("_") + 1).split("\\.")[0];

                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRequestRecord, httpPage)
                        .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                        .url(itemUrl)
                        .releaseTime(DateUtils.parseDate(releaseTimeStr,"yyyy-MM-dd HH:mm").getTime())
                        .addContentKV("content",conents.toString().trim())
                        .addContentKV("title",httpPage.getHtml().xpath("//h1[@class=\"u-title\"]/text()").get())
                        .addContentKV("author",httpPage.getHtml().xpath("//span[@class=\"m-con-source\"]/a/text()").get())
                        .build();
                return crawlerData;
            } catch (ParseException e) {
                log.error(e.getMessage(),e);
            }
        }else if(requestUrl.matches("https://edu.gmw.cn/\\d*-\\d*/\\d*/content_\\d*.htm")){
            try {

                List<String> texts = httpPage.getHtml().xpath("//div[@class=\"u-mainText\"]/p/text()").all();
                String releaseTimeStr = httpPage.getHtml().xpath("//span[@class=\"m-con-time\"]/text()").get();

                StringBuffer conents = new StringBuffer();
                for (String text : texts) {
                    conents.append(text).append(" ");
                }

                String itemUrl = httpPage.getRequest().getUrl();
                String articleKey = itemUrl.substring(itemUrl.lastIndexOf("_") + 1).split("\\.")[0];

                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRequestRecord, httpPage)
                        .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                        .url(itemUrl)
                        .releaseTime(DateUtils.parseDate(releaseTimeStr,"yyyy-MM-dd HH:mm").getTime())
                        .addContentKV("content",conents.toString().trim())
                        .addContentKV("title",httpPage.getHtml().xpath("//h1[@class=\"u-title\"]/text()").get())
                        .addContentKV("author",httpPage.getHtml().xpath("//span[@class=\"m-con-source\"]/a/text()").get())
                        .build();
                return crawlerData;
            } catch (ParseException e) {
                log.error(e.getMessage(),e);
            }
        }




        return null;

    }

    public CrawlerData washInteraction(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {

        return null;
    }


    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {

    }
}
